package com.atguigu.sparkcore.day01.singlevalue

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
 * Author atguigu
 * Date 2020/10/27 15:06
 */
object FilterDemo {
    def main(args: Array[String]): Unit = {
        val conf: SparkConf = new SparkConf().setAppName("FilterDemo").setMaster("local[2]")
        val sc: SparkContext = new SparkContext(conf)
        val list1 = List(30, 50, 70, 60, 10, 20)
        val rdd1: RDD[Int] = sc.parallelize(list1, 2)
        
        //        val rdd2 = rdd1.filter(x => x >= 60)
        //val rdd2 = rdd1.filter(_ >= 60)
        //        val rdd2 = rdd1.flatMap(x => if (x >= 60) List(x) else Nil)
        /*val rdd2 = rdd1.mapPartitions(it => {
            it.filter(_ >= 60)
        })
    */
        val rdd2 = rdd1.mapPartitions(_.filter(_ >= 60))
        rdd2.collect().foreach(println)
        sc.stop()
        
    }
}

/*
filter 可以过滤
flatMap
    碰到想过滤掉的元素, 返回一个空集合
mapPartitions
 



 */